{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Misspelling detection and correction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**NOTE**: This notebook depends upon the the Retrotech dataset. If you have any issues, please rerun the [Setting up the Retrotech Dataset](../ch04/1.setting-up-the-retrotech-dataset.ipynb) notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "sys.path.append('../..')\n",
    "from collections import defaultdict\n",
    "\n",
    "import json\n",
    "import numpy\n",
    "import pandas\n",
    "from aips import get_engine\n",
    "from aips.spark import create_view_from_collection\n",
    "from pyspark.sql import SparkSession\n",
    "spark = SparkSession.builder.appName(\"AIPS\").getOrCreate()\n",
    "engine = get_engine()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Listing 6.13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'modes': 421, 'model': 159, 'modern': 139, 'modem': 56, 'mode6': 9}\n"
     ]
    }
   ],
   "source": [
    "products_collection = engine.get_collection(\"products\")\n",
    "query = \"moden\"\n",
    "results = products_collection.spell_check(query)\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Listing 6.14"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "signals_collection = engine.get_collection(\"signals\")\n",
    "create_view_from_collection(signals_collection, \"signals\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_search_queries():\n",
    "    query = \"\"\"SELECT searches.user AS user,\n",
    "               LOWER(TRIM(searches.target)) As query\n",
    "               FROM signals AS searches WHERE searches.type = 'query'\n",
    "               GROUP BY searches.target, user\"\"\"\n",
    "    return spark.sql(query).collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "query_signals = get_search_queries()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Listing 6.15\n",
    "### Step 1: Tokenize queries and count word frequencies. \n",
    "Check word frequency distribution quantiles. The quantile will help decide cut off point for potential misspellings and corrections. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/jovyan/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
     ]
    }
   ],
   "source": [
    "from nltk import tokenize, corpus, download\n",
    "download('stopwords')\n",
    "stop_words = set(corpus.stopwords.words(\"english\"))\n",
    "\n",
    "def is_term_valid(term, minimum_length=4):\n",
    "    return (term not in stop_words and #drop stopwords\n",
    "            len(term) >= minimum_length and #only consider token length > 3, since hard to judge whether a very short token is misspelled or not\n",
    "            not term.isdigit())  # drop digit only tokens\n",
    "\n",
    "def tokenize_query(query):\n",
    "    return tokenize.RegexpTokenizer(r'\\w+').tokenize(query)\n",
    "\n",
    "def valid_keyword_occurrences(searches, tokenize=True):\n",
    "    word_list = defaultdict(int)\n",
    "    for search in searches:\n",
    "        query = search[\"query\"]\n",
    "        terms = tokenize_query(query) if tokenize else [query]\n",
    "        for term in terms:\n",
    "            if is_term_valid(term):\n",
    "                word_list[term] += 1\n",
    "    return word_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Listing 6.16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_quantiles(word_list):\n",
    "    quantiles_to_check = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]\n",
    "    quantile_values = numpy.quantile(numpy.array(list(word_list.values())),\n",
    "                                     quantiles_to_check)\n",
    "    return dict(zip(quantiles_to_check, quantile_values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0.1: 5.0,\n",
       " 0.2: 6.0,\n",
       " 0.3: 8.0,\n",
       " 0.4: 12.0,\n",
       " 0.5: 16.0,\n",
       " 0.6: 25.0,\n",
       " 0.7: 47.0,\n",
       " 0.8: 142.20000000000027,\n",
       " 0.9: 333.2000000000007}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query_signals = get_search_queries()\n",
    "word_list = valid_keyword_occurrences(query_signals, tokenize=True)\n",
    "quantiles = calculate_quantiles(word_list)\n",
    "display(quantiles)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Listing 6.17\n",
    "### Step 2: compute metadata needed for word matching. \n",
    "consider word with low count as misspelling condidates, with high count as correctly spelled candidates. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_spelling_candidates(word_list):\n",
    "    quantiles = calculate_quantiles(word_list)\n",
    "    misspellings = {\"misspelling\": [], \n",
    "                    \"misspell_counts\": [], \n",
    "                    \"misspell_length\": [],\n",
    "                    \"initial\": []}\n",
    "    corrections = {\"correction\": [], \n",
    "                   \"correction_counts\": [], \n",
    "                   \"correction_length\": [],\n",
    "                   \"initial\": []}\n",
    "    for key, value in word_list.items():\n",
    "        if value <= quantiles[0.2]: #if value == 1:  # this number based on quantile analysis and the data set, more-likely with user-behvaiour data set to be 1\n",
    "            misspellings[\"misspelling\"].append(key)\n",
    "            misspellings[\"misspell_counts\"].append(value)\n",
    "            misspellings[\"misspell_length\"].append(len(key))\n",
    "            misspellings[\"initial\"].append(key[0])\n",
    "        if value >= quantiles[0.8]:\n",
    "            corrections[\"correction\"].append(key)\n",
    "            corrections[\"correction_counts\"].append(value)\n",
    "            corrections[\"correction_length\"].append(len(key))\n",
    "            corrections[\"initial\"].append(key[0])\n",
    "    return (pandas.DataFrame(misspellings), pandas.DataFrame(corrections))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lsting 6.18\n",
    "### Step 3: Find potential matches \n",
    "based on edit distance and whether word initial is the same or not. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def good_match(word_length_1, word_length_2, edit_dist): #allow longer words have more edit distance\n",
    "    min_length = min(word_length_1, word_length_2)\n",
    "    return ((min_length < 8 and edit_dist == 1) or\n",
    "            (min_length >= 8 and min_length < 11 and edit_dist <= 2) or\n",
    "            (min_length >= 11 and edit_dist == 3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Listing 6.19\n",
    "\n",
    "### Step 4: rank potential matched corrections \n",
    "based on edit distance and correction word frequency. shorter edit distance and higher word count will be prefered. only the top one correction is selected for final matching. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk import edit_distance\n",
    "\n",
    "def calculate_spelling_corrections(word_list):\n",
    "    (misspellings, corrections) = create_spelling_candidates(word_list)\n",
    "    #Optomization: join each list based on whether they share the same initials\n",
    "    matches_candidates = pandas.merge(misspellings,\n",
    "                         corrections, on=\"initial\")    \n",
    "    matches_candidates[\"edit_dist\"] = matches_candidates.apply(\n",
    "        lambda row: edit_distance(row.misspelling,\n",
    "                                  row.correction), axis=1)\n",
    "    matches_candidates[\"good_match\"] = matches_candidates.apply(\n",
    "        lambda row: good_match(row.misspell_length,\n",
    "                               row.correction_length,\n",
    "                               row.edit_dist),axis=1)\n",
    "    \n",
    "    cols = [\"misspelling\", \"correction\", \"misspell_counts\",\n",
    "            \"correction_counts\", \"edit_dist\"]\n",
    "    matches = matches_candidates[matches_candidates[\"good_match\"]] \\\n",
    "                  .drop([\"initial\", \"good_match\"],axis=1) \\\n",
    "                  .groupby(\"misspelling\").first().reset_index() \\\n",
    "                  .sort_values(by=[\"correction_counts\", \"misspelling\"],\n",
    "                               ascending=[False, True])[cols]\n",
    "    return matches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>misspelling</th>\n",
       "      <th>correction</th>\n",
       "      <th>misspell_counts</th>\n",
       "      <th>correction_counts</th>\n",
       "      <th>edit_dist</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>181</th>\n",
       "      <td>ipad.</td>\n",
       "      <td>ipad</td>\n",
       "      <td>6</td>\n",
       "      <td>7749</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>hp tochpad</td>\n",
       "      <td>hp touchpad</td>\n",
       "      <td>6</td>\n",
       "      <td>7144</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>hp touchpad 32</td>\n",
       "      <td>hp touchpad</td>\n",
       "      <td>5</td>\n",
       "      <td>7144</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>hp toucpad</td>\n",
       "      <td>hp touchpad</td>\n",
       "      <td>6</td>\n",
       "      <td>7144</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190</th>\n",
       "      <td>iphone s4</td>\n",
       "      <td>iphone 4s</td>\n",
       "      <td>5</td>\n",
       "      <td>4642</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>193</th>\n",
       "      <td>iphone4 s</td>\n",
       "      <td>iphone 4s</td>\n",
       "      <td>5</td>\n",
       "      <td>4642</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194</th>\n",
       "      <td>iphones 4s</td>\n",
       "      <td>iphone 4s</td>\n",
       "      <td>5</td>\n",
       "      <td>4642</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>406</th>\n",
       "      <td>tochpad</td>\n",
       "      <td>touchpad</td>\n",
       "      <td>6</td>\n",
       "      <td>4019</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>407</th>\n",
       "      <td>toichpad</td>\n",
       "      <td>touchpad</td>\n",
       "      <td>6</td>\n",
       "      <td>4019</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>412</th>\n",
       "      <td>touchpaf</td>\n",
       "      <td>touchpad</td>\n",
       "      <td>5</td>\n",
       "      <td>4019</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>228</th>\n",
       "      <td>laptopa</td>\n",
       "      <td>laptop</td>\n",
       "      <td>6</td>\n",
       "      <td>3625</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229</th>\n",
       "      <td>latop</td>\n",
       "      <td>laptop</td>\n",
       "      <td>5</td>\n",
       "      <td>3625</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237</th>\n",
       "      <td>loptops</td>\n",
       "      <td>laptops</td>\n",
       "      <td>5</td>\n",
       "      <td>3435</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165</th>\n",
       "      <td>i pod tuch</td>\n",
       "      <td>ipod touch</td>\n",
       "      <td>5</td>\n",
       "      <td>2992</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>204</th>\n",
       "      <td>ipod tuch</td>\n",
       "      <td>ipod touch</td>\n",
       "      <td>6</td>\n",
       "      <td>2992</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>ipods touch</td>\n",
       "      <td>ipod touch</td>\n",
       "      <td>6</td>\n",
       "      <td>2992</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173</th>\n",
       "      <td>ipad  2</td>\n",
       "      <td>ipad 2</td>\n",
       "      <td>6</td>\n",
       "      <td>2807</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215</th>\n",
       "      <td>kimdle</td>\n",
       "      <td>kindle</td>\n",
       "      <td>5</td>\n",
       "      <td>2716</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192</th>\n",
       "      <td>iphone3</td>\n",
       "      <td>iphone</td>\n",
       "      <td>6</td>\n",
       "      <td>2599</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206</th>\n",
       "      <td>ipone</td>\n",
       "      <td>iphone</td>\n",
       "      <td>6</td>\n",
       "      <td>2599</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        misspelling   correction  misspell_counts  correction_counts  \\\n",
       "181           ipad.         ipad                6               7749   \n",
       "153      hp tochpad  hp touchpad                6               7144   \n",
       "154  hp touchpad 32  hp touchpad                5               7144   \n",
       "155      hp toucpad  hp touchpad                6               7144   \n",
       "190       iphone s4    iphone 4s                5               4642   \n",
       "193       iphone4 s    iphone 4s                5               4642   \n",
       "194      iphones 4s    iphone 4s                5               4642   \n",
       "406         tochpad     touchpad                6               4019   \n",
       "407        toichpad     touchpad                6               4019   \n",
       "412        touchpaf     touchpad                5               4019   \n",
       "228         laptopa       laptop                6               3625   \n",
       "229           latop       laptop                5               3625   \n",
       "237         loptops      laptops                5               3435   \n",
       "165      i pod tuch   ipod touch                5               2992   \n",
       "204       ipod tuch   ipod touch                6               2992   \n",
       "205     ipods touch   ipod touch                6               2992   \n",
       "173         ipad  2       ipad 2                6               2807   \n",
       "215          kimdle       kindle                5               2716   \n",
       "192         iphone3       iphone                6               2599   \n",
       "206           ipone       iphone                6               2599   \n",
       "\n",
       "     edit_dist  \n",
       "181          1  \n",
       "153          1  \n",
       "154          3  \n",
       "155          1  \n",
       "190          2  \n",
       "193          2  \n",
       "194          1  \n",
       "406          1  \n",
       "407          1  \n",
       "412          1  \n",
       "228          1  \n",
       "229          1  \n",
       "237          1  \n",
       "165          2  \n",
       "204          1  \n",
       "205          1  \n",
       "173          1  \n",
       "215          1  \n",
       "192          1  \n",
       "206          1  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "query_signals = get_search_queries()\n",
    "word_list = valid_keyword_occurrences(query_signals, tokenize=False)\n",
    "corrections = calculate_spelling_corrections(word_list)\n",
    "display(corrections.head(20))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Listing 6.20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>misspelling</th>\n",
       "      <th>correction</th>\n",
       "      <th>misspell_counts</th>\n",
       "      <th>correction_counts</th>\n",
       "      <th>edit_dist</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>181</th>\n",
       "      <td>ipad.</td>\n",
       "      <td>ipad</td>\n",
       "      <td>6</td>\n",
       "      <td>7749</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>hp tochpad</td>\n",
       "      <td>hp touchpad</td>\n",
       "      <td>6</td>\n",
       "      <td>7144</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>hp touchpad 32</td>\n",
       "      <td>hp touchpad</td>\n",
       "      <td>5</td>\n",
       "      <td>7144</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>hp toucpad</td>\n",
       "      <td>hp touchpad</td>\n",
       "      <td>6</td>\n",
       "      <td>7144</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190</th>\n",
       "      <td>iphone s4</td>\n",
       "      <td>iphone 4s</td>\n",
       "      <td>5</td>\n",
       "      <td>4642</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>193</th>\n",
       "      <td>iphone4 s</td>\n",
       "      <td>iphone 4s</td>\n",
       "      <td>5</td>\n",
       "      <td>4642</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194</th>\n",
       "      <td>iphones 4s</td>\n",
       "      <td>iphone 4s</td>\n",
       "      <td>5</td>\n",
       "      <td>4642</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>406</th>\n",
       "      <td>tochpad</td>\n",
       "      <td>touchpad</td>\n",
       "      <td>6</td>\n",
       "      <td>4019</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>407</th>\n",
       "      <td>toichpad</td>\n",
       "      <td>touchpad</td>\n",
       "      <td>6</td>\n",
       "      <td>4019</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>412</th>\n",
       "      <td>touchpaf</td>\n",
       "      <td>touchpad</td>\n",
       "      <td>5</td>\n",
       "      <td>4019</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>228</th>\n",
       "      <td>laptopa</td>\n",
       "      <td>laptop</td>\n",
       "      <td>6</td>\n",
       "      <td>3625</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229</th>\n",
       "      <td>latop</td>\n",
       "      <td>laptop</td>\n",
       "      <td>5</td>\n",
       "      <td>3625</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237</th>\n",
       "      <td>loptops</td>\n",
       "      <td>laptops</td>\n",
       "      <td>5</td>\n",
       "      <td>3435</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165</th>\n",
       "      <td>i pod tuch</td>\n",
       "      <td>ipod touch</td>\n",
       "      <td>5</td>\n",
       "      <td>2992</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>204</th>\n",
       "      <td>ipod tuch</td>\n",
       "      <td>ipod touch</td>\n",
       "      <td>6</td>\n",
       "      <td>2992</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>ipods touch</td>\n",
       "      <td>ipod touch</td>\n",
       "      <td>6</td>\n",
       "      <td>2992</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173</th>\n",
       "      <td>ipad  2</td>\n",
       "      <td>ipad 2</td>\n",
       "      <td>6</td>\n",
       "      <td>2807</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215</th>\n",
       "      <td>kimdle</td>\n",
       "      <td>kindle</td>\n",
       "      <td>5</td>\n",
       "      <td>2716</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192</th>\n",
       "      <td>iphone3</td>\n",
       "      <td>iphone</td>\n",
       "      <td>6</td>\n",
       "      <td>2599</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206</th>\n",
       "      <td>ipone</td>\n",
       "      <td>iphone</td>\n",
       "      <td>6</td>\n",
       "      <td>2599</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        misspelling   correction  misspell_counts  correction_counts  \\\n",
       "181           ipad.         ipad                6               7749   \n",
       "153      hp tochpad  hp touchpad                6               7144   \n",
       "154  hp touchpad 32  hp touchpad                5               7144   \n",
       "155      hp toucpad  hp touchpad                6               7144   \n",
       "190       iphone s4    iphone 4s                5               4642   \n",
       "193       iphone4 s    iphone 4s                5               4642   \n",
       "194      iphones 4s    iphone 4s                5               4642   \n",
       "406         tochpad     touchpad                6               4019   \n",
       "407        toichpad     touchpad                6               4019   \n",
       "412        touchpaf     touchpad                5               4019   \n",
       "228         laptopa       laptop                6               3625   \n",
       "229           latop       laptop                5               3625   \n",
       "237         loptops      laptops                5               3435   \n",
       "165      i pod tuch   ipod touch                5               2992   \n",
       "204       ipod tuch   ipod touch                6               2992   \n",
       "205     ipods touch   ipod touch                6               2992   \n",
       "173         ipad  2       ipad 2                6               2807   \n",
       "215          kimdle       kindle                5               2716   \n",
       "192         iphone3       iphone                6               2599   \n",
       "206           ipone       iphone                6               2599   \n",
       "\n",
       "     edit_dist  \n",
       "181          1  \n",
       "153          1  \n",
       "154          3  \n",
       "155          1  \n",
       "190          2  \n",
       "193          2  \n",
       "194          1  \n",
       "406          1  \n",
       "407          1  \n",
       "412          1  \n",
       "228          1  \n",
       "229          1  \n",
       "237          1  \n",
       "165          2  \n",
       "204          1  \n",
       "205          1  \n",
       "173          1  \n",
       "215          1  \n",
       "192          1  \n",
       "206          1  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "query_signals = get_search_queries()\n",
    "word_list = valid_keyword_occurrences(query_signals, tokenize=False)\n",
    "corrections = calculate_spelling_corrections(word_list)\n",
    "display(corrections.head(20))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "Up next: Chapter 7 - [Interpreting Query Intent through Semantic Search](../ch07/1.index-datasets.ipynb)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
